package com.single

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * Author atguigu
 * Date 2020/3/13 15:17
 */
object Coalesce {
  def main(args: Array[String]): Unit = {
    val conf: SparkConf = new SparkConf().setAppName("Coalesce").setMaster("local[2]")
    val sc: SparkContext = new SparkContext(conf)
    val list1 = List(30, 50, 70, 60, 10, 20)
    val rdd1: RDD[Int] = sc.parallelize(list1, 5)

    println(rdd1.getNumPartitions) // 老的分区数
    // 减少分区的时候, 默认是不会进行shuffle
    //        val rdd2 = rdd1.coalesce(10)
    val rdd2 = rdd1.coalesce(2, true) //合并分区
    println(rdd2.getNumPartitions) //新的分区数

    sc.stop()

  }
}
